{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt_tab to\n",
      "[nltk_data]     C:\\Users\\86159\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt_tab is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\86159\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "import string\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "nltk.download('punkt_tab')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_map = {\n",
    "'acc-phys': 'Accelerator Physics',\n",
    "'adap-org': 'Not available',\n",
    "'q-bio': 'Not available',\n",
    "'cond-mat': 'Not available',\n",
    "'chao-dyn': 'Not available',\n",
    "'patt-sol': 'Not available',\n",
    "'dg-ga': 'Not available',\n",
    "'solv-int': 'Not available',\n",
    "'bayes-an': 'Not available',\n",
    "'comp-gas': 'Not available',\n",
    "'alg-geom': 'Not available',\n",
    "'funct-an': 'Not available',\n",
    "'q-alg': 'Not available',\n",
    "'ao-sci': 'Not available',\n",
    "'atom-ph': 'Atomic Physics',\n",
    "'chem-ph': 'Chemical Physics',\n",
    "'plasm-ph': 'Plasma Physics',\n",
    "'mtrl-th': 'Not available',\n",
    "'cmp-lg': 'Not available',\n",
    "'supr-con': 'Not available',\n",
    "###\n",
    "\n",
    "# Added\n",
    "'econ.GN': 'General Economics',\n",
    "'econ.TH': 'Theoretical Economics',\n",
    "'eess.SY': 'Systems and Control',\n",
    "\n",
    "'astro-ph': 'Astrophysics',\n",
    "'astro-ph.CO': 'Cosmology and Nongalactic Astrophysics',\n",
    "'astro-ph.EP': 'Earth and Planetary Astrophysics',\n",
    "'astro-ph.GA': 'Astrophysics of Galaxies',\n",
    "'astro-ph.HE': 'High Energy Astrophysical Phenomena',\n",
    "'astro-ph.IM': 'Instrumentation and Methods for Astrophysics',\n",
    "'astro-ph.SR': 'Solar and Stellar Astrophysics',\n",
    "'cond-mat.dis-nn': 'Disordered Systems and Neural Networks',\n",
    "'cond-mat.mes-hall': 'Mesoscale and Nanoscale Physics',\n",
    "'cond-mat.mtrl-sci': 'Materials Science',\n",
    "'cond-mat.other': 'Other Condensed Matter',\n",
    "'cond-mat.quant-gas': 'Quantum Gases',\n",
    "'cond-mat.soft': 'Soft Condensed Matter',\n",
    "'cond-mat.stat-mech': 'Statistical Mechanics',\n",
    "'cond-mat.str-el': 'Strongly Correlated Electrons',\n",
    "'cond-mat.supr-con': 'Superconductivity',\n",
    "'cs.AI': 'Artificial Intelligence',\n",
    "'cs.AR': 'Hardware Architecture',\n",
    "'cs.CC': 'Computational Complexity',\n",
    "'cs.CE': 'Computational Engineering, Finance, and Science',\n",
    "'cs.CG': 'Computational Geometry',\n",
    "'cs.CL': 'Computation and Language',\n",
    "'cs.CR': 'Cryptography and Security',\n",
    "'cs.CV': 'Computer Vision and Pattern Recognition',\n",
    "'cs.CY': 'Computers and Society',\n",
    "'cs.DB': 'Databases',\n",
    "'cs.DC': 'Distributed, Parallel, and Cluster Computing',\n",
    "'cs.DL': 'Digital Libraries',\n",
    "'cs.DM': 'Discrete Mathematics',\n",
    "'cs.DS': 'Data Structures and Algorithms',\n",
    "'cs.ET': 'Emerging Technologies',\n",
    "'cs.FL': 'Formal Languages and Automata Theory',\n",
    "'cs.GL': 'General Literature',\n",
    "'cs.GR': 'Graphics',\n",
    "'cs.GT': 'Computer Science and Game Theory',\n",
    "'cs.HC': 'Human-Computer Interaction',\n",
    "'cs.IR': 'Information Retrieval',\n",
    "'cs.IT': 'Information Theory',\n",
    "'cs.LG': 'Machine Learning',\n",
    "'cs.LO': 'Logic in Computer Science',\n",
    "'cs.MA': 'Multiagent Systems',\n",
    "'cs.MM': 'Multimedia',\n",
    "'cs.MS': 'Mathematical Software',\n",
    "'cs.NA': 'Numerical Analysis',\n",
    "'cs.NE': 'Neural and Evolutionary Computing',\n",
    "'cs.NI': 'Networking and Internet Architecture',\n",
    "'cs.OH': 'Other Computer Science',\n",
    "'cs.OS': 'Operating Systems',\n",
    "'cs.PF': 'Performance',\n",
    "'cs.PL': 'Programming Languages',\n",
    "'cs.RO': 'Robotics',\n",
    "'cs.SC': 'Symbolic Computation',\n",
    "'cs.SD': 'Sound',\n",
    "'cs.SE': 'Software Engineering',\n",
    "'cs.SI': 'Social and Information Networks',\n",
    "'cs.SY': 'Systems and Control',\n",
    "'econ.EM': 'Econometrics',\n",
    "'eess.AS': 'Audio and Speech Processing',\n",
    "'eess.IV': 'Image and Video Processing',\n",
    "'eess.SP': 'Signal Processing',\n",
    "'gr-qc': 'General Relativity and Quantum Cosmology',\n",
    "'hep-ex': 'High Energy Physics - Experiment',\n",
    "'hep-lat': 'High Energy Physics - Lattice',\n",
    "'hep-ph': 'High Energy Physics - Phenomenology',\n",
    "'hep-th': 'High Energy Physics - Theory',\n",
    "'math.AC': 'Commutative Algebra',\n",
    "'math.AG': 'Algebraic Geometry',\n",
    "'math.AP': 'Analysis of PDEs',\n",
    "'math.AT': 'Algebraic Topology',\n",
    "'math.CA': 'Classical Analysis and ODEs',\n",
    "'math.CO': 'Combinatorics',\n",
    "'math.CT': 'Category Theory',\n",
    "'math.CV': 'Complex Variables',\n",
    "'math.DG': 'Differential Geometry',\n",
    "'math.DS': 'Dynamical Systems',\n",
    "'math.FA': 'Functional Analysis',\n",
    "'math.GM': 'General Mathematics',\n",
    "'math.GN': 'General Topology',\n",
    "'math.GR': 'Group Theory',\n",
    "'math.GT': 'Geometric Topology',\n",
    "'math.HO': 'History and Overview',\n",
    "'math.IT': 'Information Theory',\n",
    "'math.KT': 'K-Theory and Homology',\n",
    "'math.LO': 'Logic',\n",
    "'math.MG': 'Metric Geometry',\n",
    "'math.MP': 'Mathematical Physics',\n",
    "'math.NA': 'Numerical Analysis',\n",
    "'math.NT': 'Number Theory',\n",
    "'math.OA': 'Operator Algebras',\n",
    "'math.OC': 'Optimization and Control',\n",
    "'math.PR': 'Probability',\n",
    "'math.QA': 'Quantum Algebra',\n",
    "'math.RA': 'Rings and Algebras',\n",
    "'math.RT': 'Representation Theory',\n",
    "'math.SG': 'Symplectic Geometry',\n",
    "'math.SP': 'Spectral Theory',\n",
    "'math.ST': 'Statistics Theory',\n",
    "'math-ph': 'Mathematical Physics',\n",
    "'nlin.AO': 'Adaptation and Self-Organizing Systems',\n",
    "'nlin.CD': 'Chaotic Dynamics',\n",
    "'nlin.CG': 'Cellular Automata and Lattice Gases',\n",
    "'nlin.PS': 'Pattern Formation and Solitons',\n",
    "'nlin.SI': 'Exactly Solvable and Integrable Systems',\n",
    "'nucl-ex': 'Nuclear Experiment',\n",
    "'nucl-th': 'Nuclear Theory',\n",
    "'physics.acc-ph': 'Accelerator Physics',\n",
    "'physics.ao-ph': 'Atmospheric and Oceanic Physics',\n",
    "'physics.app-ph': 'Applied Physics',\n",
    "'physics.atm-clus': 'Atomic and Molecular Clusters',\n",
    "'physics.atom-ph': 'Atomic Physics',\n",
    "'physics.bio-ph': 'Biological Physics',\n",
    "'physics.chem-ph': 'Chemical Physics',\n",
    "'physics.class-ph': 'Classical Physics',\n",
    "'physics.comp-ph': 'Computational Physics',\n",
    "'physics.data-an': 'Data Analysis, Statistics and Probability',\n",
    "'physics.ed-ph': 'Physics Education',\n",
    "'physics.flu-dyn': 'Fluid Dynamics',\n",
    "'physics.gen-ph': 'General Physics',\n",
    "'physics.geo-ph': 'Geophysics',\n",
    "'physics.hist-ph': 'History and Philosophy of Physics',\n",
    "'physics.ins-det': 'Instrumentation and Detectors',\n",
    "'physics.med-ph': 'Medical Physics',\n",
    "'physics.optics': 'Optics',\n",
    "'physics.plasm-ph': 'Plasma Physics',\n",
    "'physics.pop-ph': 'Popular Physics',\n",
    "'physics.soc-ph': 'Physics and Society',\n",
    "'physics.space-ph': 'Space Physics',\n",
    "'q-bio.BM': 'Biomolecules',\n",
    "'q-bio.CB': 'Cell Behavior',\n",
    "'q-bio.GN': 'Genomics',\n",
    "'q-bio.MN': 'Molecular Networks',\n",
    "'q-bio.NC': 'Neurons and Cognition',\n",
    "'q-bio.OT': 'Other Quantitative Biology',\n",
    "'q-bio.PE': 'Populations and Evolution',\n",
    "'q-bio.QM': 'Quantitative Methods',\n",
    "'q-bio.SC': 'Subcellular Processes',\n",
    "'q-bio.TO': 'Tissues and Organs',\n",
    "'q-fin.CP': 'Computational Finance',\n",
    "'q-fin.EC': 'Economics',\n",
    "'q-fin.GN': 'General Finance',\n",
    "'q-fin.MF': 'Mathematical Finance',\n",
    "'q-fin.PM': 'Portfolio Management',\n",
    "'q-fin.PR': 'Pricing of Securities',\n",
    "'q-fin.RM': 'Risk Management',\n",
    "'q-fin.ST': 'Statistical Finance',\n",
    "'q-fin.TR': 'Trading and Market Microstructure',\n",
    "'quant-ph': 'Quantum Physics',\n",
    "'stat.AP': 'Applications',\n",
    "'stat.CO': 'Computation',\n",
    "'stat.ME': 'Methodology',\n",
    "'stat.ML': 'Machine Learning',\n",
    "'stat.OT': 'Other Statistics',\n",
    "'stat.TH': 'Statistics Theory'\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载数据集，截取对应时间的数据\n",
    "def load_filtered_data(filepath, start_year, end_year):\n",
    "    filtered_data = []\n",
    "    with open(filepath, 'r') as f:\n",
    "        for line in f:\n",
    "            paper = json.loads(line)\n",
    "            update_date = paper.get(\"update_date\")\n",
    "            if update_date:\n",
    "                paper_date = datetime.strptime(update_date, \"%Y-%m-%d\")\n",
    "                if start_year <= paper_date.year <= end_year:\n",
    "                    # Extract only the required columns\n",
    "                    filtered_data.append({\n",
    "                        \"id\": paper.get(\"id\"),\n",
    "                        \"title\": paper.get(\"title\"),\n",
    "                        \"abstract\": paper.get(\"abstract\"),\n",
    "                        \"categories\": paper.get(\"categories\"),\n",
    "                    })\n",
    "    \n",
    "    return filtered_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32md:\\Study\\Python Project\\bigdata\\arxiv_process.ipynb 单元格 4\u001b[0m line \u001b[0;36m7\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m end_year \u001b[39m=\u001b[39m \u001b[39m2024\u001b[39m\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m \u001b[39m# 加载23~24年数据\u001b[39;00m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m filtered_data \u001b[39m=\u001b[39m load_filtered_data(file_path, start_year, end_year) \n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m \u001b[39m# 转换成DataFrame格式\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mDataFrame(filtered_data)\n",
      "\u001b[1;32md:\\Study\\Python Project\\bigdata\\arxiv_process.ipynb 单元格 4\u001b[0m line \u001b[0;36m5\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m filtered_data \u001b[39m=\u001b[39m []\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mopen\u001b[39m(filepath, \u001b[39m'\u001b[39m\u001b[39mr\u001b[39m\u001b[39m'\u001b[39m) \u001b[39mas\u001b[39;00m f:\n\u001b[1;32m----> <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m     \u001b[39mfor\u001b[39;49;00m line \u001b[39min\u001b[39;49;00m f:\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m         paper \u001b[39m=\u001b[39;49m json\u001b[39m.\u001b[39;49mloads(line)\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/Study/Python%20Project/bigdata/arxiv_process.ipynb#W3sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m         update_date \u001b[39m=\u001b[39;49m paper\u001b[39m.\u001b[39;49mget(\u001b[39m\"\u001b[39;49m\u001b[39mupdate_date\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n",
      "File \u001b[1;32m<frozen codecs>:319\u001b[0m, in \u001b[0;36mdecode\u001b[1;34m(self, input, final)\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "file_path =\"./arxiv-metadata-oai-snapshot.json\"\n",
    "\n",
    "start_year = 2023\n",
    "end_year = 2024\n",
    "\n",
    "# 加载23~24年数据\n",
    "filtered_data = load_filtered_data(file_path, start_year, end_year) \n",
    "\n",
    "# 转换成DataFrame格式\n",
    "df = pd.DataFrame(filtered_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cat_text(x):\n",
    "    cat_text = ''\n",
    "    cat_list = x.split(' ')\n",
    "    for i, item in enumerate(cat_list):\n",
    "\n",
    "        cat_name = category_map[item]\n",
    "        if cat_name != 'Not available':\n",
    "            if i == 0:\n",
    "                cat_text = cat_name\n",
    "            else:\n",
    "                cat_text = cat_text + ', ' + cat_name\n",
    "    cat_text = cat_text.strip()\n",
    "\n",
    "    return cat_text\n",
    "\n",
    "\n",
    "df['cat_text'] = df['categories'].apply(get_cat_text)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 删除换行符\n",
    "def clean_text(x):\n",
    "    new_text = x.replace(\"\\n\", \" \")\n",
    "    new_text = new_text.strip()\n",
    "\n",
    "    return new_text\n",
    "\n",
    "df['title'] = df['title'].apply(clean_text)\n",
    "df['abstract'] = df['abstract'].apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "def preprocess_text(text):\n",
    "    text = text.lower()\n",
    "    text = text.translate(str.maketrans('', '', string.punctuation))\n",
    "    words = word_tokenize(text)\n",
    "    words = [word for word in words if word not in stop_words]\n",
    "    return ' '.join(words)\n",
    "\n",
    "df['processed_abstract'] = df['abstract'].apply(preprocess_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df['text'] = df['title'] + ' {title} ' + df['abstract']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_json(\"processd_data.json\", orient=\"records\", lines=True, force_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunk_list = list(df['text'])\n",
    "\n",
    "\n",
    "arxiv_id_list = list(df['id'])\n",
    "cat_list = list(df['cat_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "\n",
    "embeddings = model.encode(chunk_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez_compressed('compressed_array.npz', array_data=embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_embeddings = np.load('compressed_array.npz')\n",
    "\n",
    "\n",
    "loaded_embeddings = loaded_embeddings['array_data']\n",
    "embeddings = loaded_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import faiss\n",
    "\n",
    "embed_length = embeddings.shape[1]\n",
    "\n",
    "num_centroids = 5\n",
    "\n",
    "quantizer = faiss.IndexFlatL2(embed_length)\n",
    "\n",
    "index = faiss.IndexIVFFlat(quantizer, embed_length, num_centroids)\n",
    "\n",
    "index.train(embeddings)\n",
    "\n",
    "index.add(embeddings)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "my_python",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
